#!/usr/bin/env perl
# Simple check of the MPI bandwidths and latencies using wrekaMPI.py

# Basile Clout,
# November 2007

use Getopt::Long;

use strict;
use warnings;

sub usage(){

  print <<EOF;

Usage: $0 [-f file containing the list of nodes] [-h print help and
exit] [-u user] [-l logfile] [--fast] <list of nodes>

EOF
  exit;
}

my $tmp_file = "/tmp/tmp_list.txt";
my $file;
my $help;
my $user="root";
my @hosts;
my $ssh = "ssh";
my $scp = "scp";
my $fast;
my $logfile = "chkMPI.log";

GetOptions("f=s"=>\$file, "h"=>\$help, "u=s"=>\$user, "l=s"=>\$logfile, "fast"=>\$fast);

my $wrekaMPI_bin = "../daemons/wrekaMPI/wreka_MPI";

my %hash;

usage() if ($help);

if ($#ARGV < 0 and !$file){
  print "Please enter at least one node, either on the command line or in a file\n";
  usage();
}

if ($#ARGV > -1){
  @hosts = sort @ARGV;
}else{
  open(HOSTS, "<$file") or die("$0: Impossible to open file $file: $!");
  my @oldhosts = <HOSTS>;
  @hosts = sort split /\s/, $oldhosts[$#oldhosts];
  close(HOSTS);
}
#print "hosts: ", join(' & ', @hosts), "\n";
my $length = scalar @hosts;

open LOG, ">$logfile" or die "Impossible to create logfile $logfile: $!";
my ($sec, $min, $hrs, $day, $month, $year) = (localtime) [0,1,2,3,4,5];
my $date = sprintf("%04d-%02d-%02d %d:%d:%d\n", $year+1900, $month+1, $day, $hrs, $min, $sec);
print LOG "#Basile Clout, $date \n\n";

print LOG "Number of nodes: $length\n";
print "$length mpi \n";

my $mylength;
my $cnt = 0;

# Real loop
foreach my $i (0..$#hosts){

  my @tmp_list = @hosts;
  if ($fast){
    for(my $j=0;$j<$cnt;$j++){
      my $shifted = shift @tmp_list;
      $hash{$shifted}=['  -','  -'];
    }
  }

  # Create temp host file
  printf "\t";
  open TMP, ">$tmp_file" or die "Impossible to create temporary list of files $tmp_file: $!";
  foreach my $node (@tmp_list){
    chomp $node;
    print TMP "${node}\n";
    if ($i==0){
      printf "%s\t", $node;
    }
  }
  close(TMP);
  print "\n";
  
  $mylength = scalar @tmp_list;
  my $node = $hosts[$i];
  my $master=$i;
  $master=0 if ($fast);

  if (!($fast and $i==$#hosts)){
    # Send the hosts list to all the involved node:
    my $scp_order = "$scp $tmp_file ${node}:/tmp ";
    open SCP, "$scp_order |" or die "ERROR: Impossible to send hosts file to node $node: $!";
    my @scp_content = <SCP>;
    print LOG "scp hosts file (order = $scp_order) to $node: ", join('', @scp_content), "\n";
    
    my $order = "$ssh ${node} \"sudo -H -u ${user} /usr/local/bin/mpiexec -machinefile $tmp_file -n $mylength $wrekaMPI_bin -m $master\"";
    print LOG "$order\n";
    
    open ORDER, "$order |" or die "ERROR: Impossible to order $order: $!";
    
    my @content = <ORDER>;
    print LOG join('', @content);
    
    foreach my $line (@content){
      if ($line =~ /^\s(\d+)\s(\d+)(\s\d+){9}\s+$/){
	die "ERROR: number of nodes does not correspond ($2, $mylength)" if ($2 != $mylength);
      }elsif ($line =~ /^\d+\s\d+(\s\d+\.\d+){8}\s+$/){
	my @arr = split /\s/, $line;
	$hash{$hosts[$arr[1]+$cnt]}=[ $arr[6], $arr[3] ];
      }
    }
  }
  $hash{$node} = [ "  -", "  -" ];
  
  print "$node\t";
  foreach (@hosts) {
    print " $hash{$_}[0]\t";
  }
  print "\n\t";
  foreach (@hosts) {
    print " $hash{$_}[1]\t";
  }
  $cnt++ if ($fast);
}
print "\n";
